{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "from glob import glob\n",
    "import os\n",
    "from skimage.io import imread\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import BatchNormalization, Conv2D, UpSampling2D, Lambda\n",
    "\n",
    "from skimage.morphology import label # label regions\n",
    "from skimage.morphology import closing, opening, disk\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dsb_data_dir = os.path.join('..', 'input')\n",
    "stage_label = 'stage1'\n",
    "epoch_size = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# read in lables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_labels = pd.read_csv(os.path.join(dsb_data_dir,'{}_train_labels.csv'.format(stage_label)))\n",
    "train_labels['EncodedPixels'] = train_labels['EncodedPixels'].map(lambda ep: [int(x) for x in ep.split(' ')])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load in all Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>ImageId</th>\n",
       "      <th>ImageType</th>\n",
       "      <th>TrainingSplit</th>\n",
       "      <th>Stage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15063</th>\n",
       "      <td>./input/stage1_train/ea00f5a91ca75e745d675201c...</td>\n",
       "      <td>ea00f5a91ca75e745d675201cc62d7db266f8e2787033e...</td>\n",
       "      <td>masks</td>\n",
       "      <td>train</td>\n",
       "      <td>stage1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9448</th>\n",
       "      <td>./input/stage1_train/a022908f1b7880838dbc0411e...</td>\n",
       "      <td>a022908f1b7880838dbc0411e50828e64b4f5e0263afdf...</td>\n",
       "      <td>masks</td>\n",
       "      <td>train</td>\n",
       "      <td>stage1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    path  \\\n",
       "15063  ./input/stage1_train/ea00f5a91ca75e745d675201c...   \n",
       "9448   ./input/stage1_train/a022908f1b7880838dbc0411e...   \n",
       "\n",
       "                                                 ImageId ImageType  \\\n",
       "15063  ea00f5a91ca75e745d675201cc62d7db266f8e2787033e...     masks   \n",
       "9448   a022908f1b7880838dbc0411e50828e64b4f5e0263afdf...     masks   \n",
       "\n",
       "      TrainingSplit   Stage  \n",
       "15063         train  stage1  \n",
       "9448          train  stage1  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_images = glob(os.path.join(dsb_data_dir, 'stage1_*', '*', '*', '*'))\n",
    "img_df = pd.DataFrame({'path': all_images})\n",
    "img_id = lambda in_path: in_path.split('/')[-3]\n",
    "img_type = lambda in_path: in_path.split('/')[-2]\n",
    "img_group = lambda in_path: in_path.split('/')[-4].split('_')[1]\n",
    "img_stage = lambda in_path: in_path.split('/')[-4].split('_')[0]\n",
    "img_df['ImageId'] = img_df['path'].map(img_id)\n",
    "img_df['ImageType'] = img_df['path'].map(img_type)\n",
    "img_df['TrainingSplit'] = img_df['path'].map(img_group)\n",
    "img_df['Stage'] = img_df['path'].map(img_stage)\n",
    "img_df.sample(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Training Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_df = img_df.query('TrainingSplit==\"train\"')\n",
    "train_rows = []\n",
    "group_cols = ['Stage', 'ImageId']\n",
    "for n_group, n_rows in train_df.groupby(group_cols):\n",
    "    c_row = {col_name: col_value for col_name, col_value in zip(group_cols, n_group)}\n",
    "    c_row['masks'] = n_rows.query('ImageType == \"masks\"')['path'].values.tolist()\n",
    "    c_row['images'] = n_rows.query('ImageType == \"images\"')['path'].values.tolist()\n",
    "    train_rows += [c_row]\n",
    "train_img_df = pd.DataFrame(train_rows)    \n",
    "IMG_CHANNELS = 3\n",
    "def read_and_stack(in_img_list):\n",
    "    return np.sum(np.stack([imread(c_img) for c_img in in_img_list], 0), 0)/255.0\n",
    "train_img_df['images'] = train_img_df['images'].map(read_and_stack).map(lambda x: x[:,:,:IMG_CHANNELS])\n",
    "train_img_df['masks'] = train_img_df['masks'].map(read_and_stack).map(lambda x: x.astype(int))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# simple CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "NormalizeInput (BatchNormali (None, None, None, 3)     12        \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, None, None, 8)     224       \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, None, None, 8)     584       \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, None, None, 16)    1168      \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, None, None, 16)    2320      \n",
      "_________________________________________________________________\n",
      "conv2d_5 (Conv2D)            (None, None, None, 32)    4640      \n",
      "_________________________________________________________________\n",
      "conv2d_6 (Conv2D)            (None, None, None, 16)    528       \n",
      "_________________________________________________________________\n",
      "conv2d_7 (Conv2D)            (None, None, None, 1)     17        \n",
      "=================================================================\n",
      "Total params: 9,493\n",
      "Trainable params: 9,487\n",
      "Non-trainable params: 6\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "simple_cnn = Sequential()\n",
    "simple_cnn.add(BatchNormalization(input_shape = (None, None, IMG_CHANNELS), \n",
    "                                  name = 'NormalizeInput'))\n",
    "simple_cnn.add(Conv2D(8, kernel_size = (3,3), padding = 'same'))\n",
    "simple_cnn.add(Conv2D(8, kernel_size = (3,3), padding = 'same'))\n",
    "# use dilations to get a slightly larger field of view\n",
    "simple_cnn.add(Conv2D(16, kernel_size = (3,3), dilation_rate = 2, padding = 'same'))\n",
    "simple_cnn.add(Conv2D(16, kernel_size = (3,3), dilation_rate = 2, padding = 'same'))\n",
    "simple_cnn.add(Conv2D(32, kernel_size = (3,3), dilation_rate = 3, padding = 'same'))\n",
    "\n",
    "# the final processing\n",
    "simple_cnn.add(Conv2D(16, kernel_size = (1,1), padding = 'same'))\n",
    "simple_cnn.add(Conv2D(1, kernel_size = (1,1), padding = 'same', activation = 'sigmoid'))\n",
    "simple_cnn.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# define Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "smooth = 1.\n",
    "def dice_coef(y_true, y_pred):\n",
    "    y_true_f = K.flatten(y_true)\n",
    "    y_pred_f = K.flatten(y_pred)\n",
    "    intersection = K.sum(y_true_f * y_pred_f)\n",
    "    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n",
    "def dice_coef_loss(y_true, y_pred):\n",
    "    return -dice_coef(y_true, y_pred)\n",
    "simple_cnn.compile(optimizer = 'adam', \n",
    "                   loss = dice_coef_loss, \n",
    "                   metrics = [dice_coef, 'acc', 'mse'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "670/670 [==============================] - 557s 832ms/step - loss: -0.7116 - dice_coef: 0.7116 - acc: 0.8959 - mean_squared_error: 0.0982\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x11c81a080>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def simple_gen():\n",
    "    while True:\n",
    "        for _, c_row in train_img_df.iterrows():\n",
    "            yield np.expand_dims(c_row['images'],0), np.expand_dims(np.expand_dims(c_row['masks'],-1),0)\n",
    "\n",
    "simple_cnn.fit_generator(simple_gen(), \n",
    "                         steps_per_epoch=train_img_df.shape[0],\n",
    "                        epochs = epoch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65 images to process\n"
     ]
    }
   ],
   "source": [
    "test_df = img_df.query('TrainingSplit==\"test\"')\n",
    "test_rows = []\n",
    "group_cols = ['Stage', 'ImageId']\n",
    "for n_group, n_rows in test_df.groupby(group_cols):\n",
    "    c_row = {col_name: col_value for col_name, col_value in zip(group_cols, n_group)}\n",
    "    c_row['images'] = n_rows.query('ImageType == \"images\"')['path'].values.tolist()\n",
    "    test_rows += [c_row]\n",
    "test_img_df = pd.DataFrame(test_rows)    \n",
    "\n",
    "test_img_df['images'] = test_img_df['images'].map(read_and_stack).map(lambda x: x[:,:,:IMG_CHANNELS])\n",
    "print(test_img_df.shape[0], 'images to process')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_img_df['masks'] = test_img_df['images'].map(lambda x: simple_cnn.predict(np.expand_dims(x, 0))[0, :, :, 0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check RLE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rle_encoding(x):\n",
    "    '''\n",
    "    x: numpy array of shape (height, width), 1 - mask, 0 - background\n",
    "    Returns run length as list\n",
    "    '''\n",
    "    dots = np.where(x.T.flatten()==1)[0] # .T sets Fortran order down-then-right\n",
    "    run_lengths = []\n",
    "    prev = -2\n",
    "    for b in dots:\n",
    "        if (b>prev+1): run_lengths.extend((b+1, 0))\n",
    "        run_lengths[-1] += 1\n",
    "        prev = b\n",
    "    return run_lengths\n",
    "\n",
    "def prob_to_rles(x, cut_off = 0.5):\n",
    "    lab_img = label(x>cut_off)\n",
    "    if lab_img.max()<1:\n",
    "        lab_img[0,0] = 1 # ensure at least one prediction per image\n",
    "    for i in range(1, lab_img.max()+1):\n",
    "        yield rle_encoding(lab_img==i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate the RLEs for a Train Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, train_rle_row = next(train_img_df.tail(5).iterrows()) \n",
    "train_row_rles = list(prob_to_rles(train_rle_row['masks']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Take the RLEs from the CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tl_rles = train_labels.query('ImageId==\"{ImageId}\"'.format(**train_rle_row))['EncodedPixels']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Matches: 360, Mismatches: 0, Accuracy: 100.0%\n"
     ]
    }
   ],
   "source": [
    "match, mismatch = 0, 0\n",
    "for img_rle, train_rle in zip(sorted(train_row_rles, key = lambda x: x[0]), \n",
    "                             sorted(tl_rles, key = lambda x: x[0])):\n",
    "    for i_x, i_y in zip(img_rle, train_rle):\n",
    "        if i_x == i_y:\n",
    "            match += 1\n",
    "        else:\n",
    "            mismatch += 1\n",
    "print('Matches: %d, Mismatches: %d, Accuracy: %2.1f%%' % (match, mismatch, 100.0*match/(match+mismatch)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate RLE for all the masks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clean_img(x):\n",
    "    return opening(closing(x, disk(1)), disk(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_img_df['rles'] = test_img_df['masks'].map(clean_img).map(lambda x: list(prob_to_rles(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1915 regions found for 65 images\n"
     ]
    }
   ],
   "source": [
    "out_pred_list = []\n",
    "for _, c_row in test_img_df.iterrows():\n",
    "    for c_rle in c_row['rles']:\n",
    "        out_pred_list+=[dict(ImageId=c_row['ImageId'], \n",
    "                             EncodedPixels = ' '.join(np.array(c_rle).astype(str)))]\n",
    "out_pred_df = pd.DataFrame(out_pred_list)\n",
    "print(out_pred_df.shape[0], 'regions found for', test_img_df.shape[0], 'images')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "out_pred_df[['ImageId', 'EncodedPixels']].to_csv('predictions.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "TF36",
   "language": "python",
   "name": "tf36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
